#!/usr/bin/env python
# coding: utf-8

from google.appengine.ext import db

import module.utility

import logging
import extlib.BeautifulSoup
import urllib2
import data.model
import datetime

class CrawlProducts:
    @staticmethod
    def convert_string(str_value):
        return str_value.encode('utf-8').decode('utf-8')

    @staticmethod
    def is_exist(item):
        #######################################
        # TODO need to be check uniqueness
        q = db.GqlQuery("SELECT * FROM Products WHERE title = :1", 
                        CrawlProducts.convert_string(item.title.string))                    
        result = q.fetch(1)
        if not result:
            return False
        else:
            return True
        
    @staticmethod
    def cralwer_social_data(social_provider, social_code, xml_url):
        # optimize using google api
        try:
            retData = urllib2.urlopen(xml_url).read()
        except urllib2.HTTPError, e:
            logging.error("Http Error: %d" % e.code)
            print "Http Error: %d" % e.code
        except urllib2.URLError, e:
            logging.error("Network error: %s" % e.reason.args[1])
            print "Network error: %s" % e.reason.args[1]
    
        insert_count = 0
        soup = extlib.BeautifulSoup.BeautifulSoup(retData)
        for item in soup.findAll('item'):
            if CrawlProducts.is_exist(item):
                if item.sell_count != None:
                    # Update sell count
                    q = db.GqlQuery("SELECT * FROM Products WHERE title = :1", 
                                    CrawlProducts.convert_string(item.title.string))
                    p = q.fetch(1)[0]
                    if datetime.date(p.latest_update.year, p.latest_update.month, p.latest_update.day) == datetime.date.today():
                        continue
                    
                    l = p.sell_list
                    l.append(int(item.sell_count.string))
                    p.sell_list = l
                    
                    # Update sell_delta
                    days = (datetime.date.today() - p.latest_update).days
                    if days == 0:
                        p.sell_delta = int(item.sell_count.string)
                    else:
                        p.sell_delta  = int(sum(l)/days)
                    p.put()
                    continue
                
            p = data.model.Products()
                   
            if item.title != None:
                p.title = CrawlProducts.convert_string(item.title.string)
            
            if item.time_start != None:
                p.time_start = datetime.datetime.strptime(item.time_start.string, "%Y-%m-%d %H:%M:%S")
                
            if item.time_end != None:
                p.time_end = datetime.datetime.strptime(item.time_end.string, "%Y-%m-%d %H:%M:%S")
                # 0 - Available, 1-Expired  
                if (p.time_end - datetime.datetime.today()):
                    p.is_expired = 0
                else:
                    p.is_expired = 1
                    
            if item.link.nextSibling.string != None: 
                p.link = CrawlProducts.convert_string(item.link.nextSibling.string)
            
            if item.price_original != None:
                p.price_original = int(item.price_original.string)
                
            if item.price_now != None:
                p.price_now = int(item.price_now.string)
                
            if item.sale_percent != None:
                p.sale_percent = int(item.sale_percent.string)
            if item.photo1 != None:
                p.photo_url = CrawlProducts.convert_string(item.photo1.string)
                        
            if item.area != None:
                p.area = CrawlProducts.convert_string(item.area.string)
          
            if item.type != None:            
                p.type = CrawlProducts.convert_string(item.type.string)
                
            if item.category != None:
                p.category = CrawlProducts.convert_string(item.category.string)
            
			# if item.latitude != None:
                # p.latitude = CrawlProducts.convert_string(item.latitude.string)
            
			# if item.longitude != None:
                # p.longitude = CrawlProducts.convert_string(item.longitude.string)
            
            if social_provider != None:
                p.social_provider = social_provider
            if social_code != None:
                p.social_provider_code = social_code
                
            if item.sell_count != None:
                sell_list = [int(item.sell_count.string)]
                p.sell_list = sell_list
                p.sell_delta = int(item.sell_count.string)
            else:
                p.sell_list = []
                p.sell_delta = 0

            keywords = []
            keywords.extend(module.utility.Utility.make_keywords(item.title.string))
            p.keywords = keywords
            
            p.latest_update = datetime.datetime.now()
            
            p.put()
            
            insert_count+=1
        
        return insert_count 
